'''
Created on 2009-10-29

@author: lz
'''

from lxml import etree
from cStringIO import StringIO
import urllib2
import re

class CBulletin(object):
    pass

def test_bulletins_border():
    """
    up to 63
    """
    import urllib2
    
    pattern = 'http://www.microsoft.com/technet/security/Bulletin/MS09-%03d.mspx'
    for i in range(1, 100):
        conn = urllib2.urlopen(pattern % (i,))
        conn.close()

def rebuild_matrix(martrix):
    for i in range(len(martrix)):
        for j in range(len(martrix[i])):
            if martrix[i][j][0] == 'text':
                if martrix[i][j][1] == 'See row below':
                    martrix[i][j] = martrix[i+1][j]
                if martrix[i][j][1] == 'See row above':
                    martrix[i][j] = martrix[i-1][j]
    return martrix

def parse_title(buffer):
    p = re.compile('MS\d\d-\d\d\d')
    bulletins_num = p.search(buffer).group()
    i = buffer.index(bulletins_num) + len(bulletins_num) + 3
    return bulletins_num, buffer[i:]
    
def parsesection(content):
    

    
    parser = etree.HTMLParser()
    tree   = etree.parse(StringIO(content), parser)
    
    """
    u'Microsoft Security Bulletin MS09-008 \u2013 Important: Vulnerabilities in DNS and WINS Server Could Allow Spoofing (962238)'
    """

    #title method 1
    html_title = tree.findtext('head/title')
    bulletins_num, description = parse_title(html_title)
        
#    title method 2
#    root = tree.getroot()
#    rootchilds = root.getchildren() 
#    head = rootchilds[0]
#    title = head.find('title')
#    titletext = title.text
    
    dataTable = None
    tables = tree.getiterator('table')
    
    # find first table which id length is 3 such as EED,EVD,EVC,ENC
    for item in tables:
        table_id = item.attrib.get('id')
        if table_id and len(table_id) == 3:
            dataTable = item
            break
    
    childs = dataTable.getchildren()
    #------------thead--------------
    thead = childs[0]
    tr = childs[0].getchildren()[0]
    tds = tr.getchildren()
    lables = []
    
    for item in tds:
        lables.append(item.text)
    print lables
    #------------tbody--------------
    tbody = childs[1]
    trs = tbody.getchildren()
    result = []
    
    for tds in trs:
        line_result = []
        for td in tds:
            try:
                tdc = td.getchildren()[0] #p
                """
                3 case
                1)td1c.getchildren()
                [<Element a at 11966f0>, <Element br at 1196810>]
                
                <a href="http://www.microsoft.com/downloads/details.aspx?familyid=110354f7-5ece-4c4d-b563-3adba6ac0116">DNS server on Microsoft Windows 2000 Server Service Pack 4</a><br>(961063)
                td1cc1.attrib = {'href': 'http://www.microsoft.com/downloads/details.aspx?familyid=110354f7-5ece-4c4d-b563-3adba6ac0116'}
                td1cc1.text = 'DNS server on Microsoft Windows 2000 Server Service Pack 4'
                
                2)td2c.getchildren()
                0
                td2c.text = See row below
                
                3)td4c.getcihldren()
                [<Element a at 1190510>]
                td4cc = td4c.getcihldren()[0]
                td4cc.attrib = {'href': 'http://go.microsoft.com/fwlink/?LinkID=119620'}
                td4cc.text = 'MS08-037'
                """
                type, text, link = None, None, None
                p_child_count = len(tdc.getchildren())
                if (p_child_count) == 0:
                    type = 'text'
                    text = tdc.text
                elif (p_child_count) == 1:
                    tdcc = tdc.getchildren()[0]
                    type = 'link'
                    text = tdcc.text
                    link = tdcc.attrib.get('href')
                elif (p_child_count) == 2:
                    tdcc = tdc.getchildren()[0]
                    tdbr = tdc.getchildren()[1]
                    type = 'link'
                    text = tdcc.text + tdbr.tail
                    link = tdcc.attrib.get('href')
                line_result.append((type, text, link))
            except:
                pass
        if line_result:
            result.append(line_result)

    result = rebuild_matrix(result)
    for line in result:
        print line
    
def parsesection_batch():
    from os.path import join
    from os import listdir
    p_url = 'http://www.microsoft.com/technet/security/Bulletin/MS09-%03d.mspx'
    
    p_file = r'e:\workspace_ganymede\beta_vul\src\ms\MS09-%03d.html'
    
#    conn = urllib2.urlopen(p_url % (i,))
#    content = conn.read()
#    conn.close()
        
    for i in range(1, 63):
        filename = p_file % (i,)
        f = open(filename, 'rb')
        content = f.read()
        f.close()
    
        print i
        result = parsesection(content)

def test():
    filename = r'e:\workspace_ganymede\beta_vul\src\ms-bulletin\sample\MS09-008.htm'
    content = open(filename, 'rb').read()
    parsesection(content)
    print 'done'   
    
if __name__ == '__main__':
    parsesection_batch()
